#ifndef __PAGELIB_H__
#define __PAGELIB_H__

#include "SplitTool.hpp"
#include "Configuration.hpp"
#include "tinyxml2.h"
#include <dirent.h>
#include <iostream>
#include <fstream>
#include <regex>
#include <vector>
#include <string>
#include <map>

using namespace std;
using WordSegmentation = se::SplitToolCppJieba;
using std::ofstream;
using std::regex;

namespace se
{

class DirScanner
{
public:
    std::vector<std::string> & getFiles();
    void traverse(std::string dir);

private:
    std::vector<std::string> _files;
};


class FileProcessor
{
public:
    std::string process(std::string fileName);

private:
    std::string _titleFeature;
};


class PageLib
{
public:
    PageLib(DirScanner & dirScanner);
    PageLib(Configuration & conf, DirScanner & dirScanner, FileProcessor & fileProcessor);//构造函数
    void create();//创建网页库
    void store();//存储网页库和位置偏移库

private:
    DirScanner & _dirScanner;//目录扫描对象的引用
    std::vector<std::string> _files;//存放格式化之后的网页的容器
    std::map<int, std::pair<int, int>> _offsetLib;//存放每篇文档在网页库的位置信息
};

struct RssItem
{
    string title;
    string link;
    string description;
};

void filter(string &s)
{
    regex table_re1("<span.*?>.*?</span>");
    regex table_re2("<[^<]*>");
    s = regex_replace(s,table_re1,"");
    s = regex_replace(s,table_re2,"");
}

}//end of namespace se

#endif

